package freebase;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.*;
import java.io.*;

class Filter {
	static int MATCHNUMFILTER = 10;
	static int DISTINCTVALUEFILER = 5;
	static double MISSRATELIMIT = 0.6;

	public void caseNumFilter(PropMatch pm) {
		if (pm.matches.size() <= MATCHNUMFILTER) {
			pm.passfilter = false;
		}
	}

	public void distinctValueFilter(PropMatch pm) {
		HashSet<String> distinctValues = new HashSet<String>();
		for (OneMatch om : pm.matches) {
			distinctValues.addAll(om.matched_values);
		}
		if (distinctValues.size() < DISTINCTVALUEFILER) {
			pm.passfilter = false;
		}
	}

	public void missrateFilter(PropMatch pm) {
		int total = 0;
		int goal = 0;
		for (OneMatch om : pm.matches) {
			total += om.fb_values.length;
			goal += om.matched_values.size();
		}
		if (goal * 1.0 / total < MISSRATELIMIT) {
			pm.passfilter = false;
		}
	}
}

class PropMatch {
	ArrayList<OneMatch> matches = new ArrayList<OneMatch>();
	ArrayList<String> org = new ArrayList<String>();
	boolean passfilter = true;

	public PropMatch(String filename) throws Exception {
		BufferedReader br = new BufferedReader(new InputStreamReader(
				new FileInputStream(filename), "utf-8"));
		String line = "";
		while ((line = br.readLine()) != null) {
			org.add(line);
			OneMatch om = new OneMatch(line);
			this.matches.add(om);
		}
	}

	public void wrtToFile(String filename) throws Exception {
		if (!passfilter)
			return;
		OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(
				filename), "utf-8");
		for (String line : org) {
			osw.write(line + "\n");
		}
		osw.close();
	}

}

class OneMatch {
	int wpid;
	String title;
	int senid;
	String type;

	String[] fb_values;
	ArrayList<String> matched_values = new ArrayList<String>();
	ArrayList<Integer> matched_pos = new ArrayList<Integer>();

	String sentence;

	public OneMatch(String line) {
		String[] abc = line.split("\t");
		this.wpid = Integer.parseInt(abc[0]);
		this.title = abc[1];
		this.senid = Integer.parseInt(abc[2]);
		this.type = abc[3];
		String[] pos_str = abc[4].substring(1, abc[4].length() - 1)
				.split("\\,");
		int[] pos = new int[pos_str.length];
		for (int i = 0; i < pos.length; i++) {
			pos[i] = (int) Math.round(Double.parseDouble(pos_str[i]));
		}
		this.fb_values = abc[5].split("\\|");
		if (fb_values.length != pos.length) {
			System.err.println("" + abc[4] + "\t" + abc[5]);
		}
		try {
			for (int i = 0; i < pos.length; i++) {
				if (pos[i] >= 0) {
					matched_pos.add(pos[i]);
					matched_values.add(fb_values[i]);
				}
			}
		} catch (Exception e) {

		}
		sentence = abc[6];

	}

}

public class FilterTraining {

	/**
	 * @param args
	 */
	public static void main(String[] args) throws Exception {
		// TODO Auto-generated method stub
		File dir = new File(args[0]);
		String[] flist = dir.list();
		for (String a : flist) {
			System.out.println(a);
			String file = dir + "/" + a;
			PropMatch pm = new PropMatch(file);
			Filter ft = new Filter();
			ft.caseNumFilter(pm);
			ft.distinctValueFilter(pm);
			ft.missrateFilter(pm);
			pm.wrtToFile(args[1] + "/" + a);
		}

	}

}
